{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from deeptables.datasets import dsutils\n",
    "from deeptables.eda.utils import columns_info\n",
    "from deeptables.models import make_experiment\n",
    "from sklearn.model_selection import train_test_split\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((8000, 18), (2000, 18))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = dsutils.load_bank().head(10000)\n",
    "train_data, test_data = train_test_split(df, test_size=0.2, random_state=42)\n",
    "\n",
    "train_data.shape, test_data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyze data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# columns_info(train_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make experiment and run it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "### Input Data"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X_train.shape</th>\n",
       "      <th>y_train.shape</th>\n",
       "      <th>X_eval.shape</th>\n",
       "      <th>y_eval.shape</th>\n",
       "      <th>X_test.shape</th>\n",
       "      <th>Task</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(8000, 17)</td>\n",
       "      <td>(8000,)</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>binary(2)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  X_train.shape y_train.shape X_eval.shape y_eval.shape X_test.shape  \\\n",
       "0    (8000, 17)       (8000,)         None         None         None   \n",
       "\n",
       "        Task  \n",
       "0  binary(2)  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiQAAAElCAYAAADUR/lIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAxOAAAMTgF/d4wjAAAcBklEQVR4nO3df7RdZX3n8fcnJGAxiiUGqcWQSERK1ao1VGdo0TqDdKqlogPYgqN1MEoY6lCmzFKX40ynXYyMLEdKYVAZNLooP2wpbRnsVHFFllChiIoFkyDhGpWUJqANrYQk3/nj7KuH601yzs3NfXLufb/W2uuevb/P2fvZm5t7Pjz7OeekqpAkSWppXusOSJIkGUgkSVJzBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRJoDklyV5JPTvM+lSSrJ8m79LUk2TucxJjtOK0neneTvu768smVfpNnIQCKNsCSf714gK8k/J7m/Cx8/N6HpbwOrBtjf/CFecL8F/BTwwNAd3/Xx/32SDfv6OMNKciTw34G3d335Yqu+SLOVgUQafR+i9yL5fOBtwALgjiSvG29QVd+rqu9N1wGTHFRVO6rqoaraMV37ncxMHWcPlgEB/qzry7aGfZFmJQOJNPoe614kx6rq81X1m8AngMuSLIAfv2WT5F1JHkjyeJKNSd7fldZ3P2/pRkqu6tp/Psn/TPKRJN8HPrirWylJzkzyrSRbu/YH9tUqyb/qW//hPrpRmY8AR/aN+rxysuMkeXOS9V3/v5bkV/pqr+zavzrJ3yX5xyQ3JPnJ3V3EJP+p6/fjSW5Pcly3/S3ALV2znUl+7Ps2krw8yQ8mHiPJF/quraTdMJBIs9MlwE8DL51YSLIC+K/AO4DnAafyoyDy8u7nG+iNuvx231NXAvd3+/zgLo67iN4ozWuB1wO/Crx7wD5/EfgdYGN37ElvjST5F8CVwIeBFwF/CtyQZOmEpu8F3gK8Cnhhtz6pJL8BvB/4z8CLga8CNyV5OnANvWtEX7+epKpup3dLabwdSZYB/xJYvetTljTOQCLNTvd1P5dOUlsCPAR8thtV+WJVjY+e/EP3c0s36tJ/m+fOqrqwqtZX1a7mczwFeEdVfaWq/h/wPuCcQTrc3Qb5PjB+i2ZXt0bOBf6kqj5cVd+oqvcBX+bH58hcUFVfqqo7gY8CJ+zm8OcCl1bVp6rqXuBs4J+BM6vqn4FHuj4+VFUP7WIfHwfe3Ld+JvDFqrp/tycuCTCQSLNVup+TfZ33X3fb709yeZJfTZJJ2k305QHa/GNV3de3/iVgUZJFAzx3UM8Hbp+w7bZue7+v9T1+CDhs0H1W1Xbgzkn2uTufAH4hyVHd+hndNkkDMJBIs9Mx3c8NEwvdqMeLgHcC2+jd/vizAfb5TwO0mSwATaz3h58FA+xzokHCE8ATE467T//eVdV36IW9M5O8AngOcO2+PKY0mxhIpNnpP9B7u+xdkxWraltV3VRV5wKvA16X5DBgB7ATOGCKx316kv5RhRXA5qra3K0/DBzeV3/hhOc/McCx7+NHc13GvYIf3aaaim/07zPJfOBlU9jn/6E3MvJm4MaqenQv+iTNKfNbd0DSXntqksOBA4HnAmcBbwRe3916eJIkrwWOBNYAjwGn0Zs7srmqKsm3gF9O8jXgn6pq6xB9+QG9d/f8R+CZ9CbPXtpXXwP8dpIv05sAO3Gi6YPAs5K8jN7ozmRvVf4wsCbJOcBf0QsALwFOH6KfE/0v4CNJ7qYX4s4DfgIY9sPkbgAupzex9/V70R9pznGERBp97wK+C6yld/vlCWBFVd20i/aP0gshX6D3bpLjgNf2fc7H7wK/2e3zD4fsy2Z68yZuAm4EPgP8QV/9d4B/BP4GuJjeO1v6rQH+mN6tj4fpvUvlSarqi8Bv0Tvve+i98P96VW0Ysq/9+7yaXnj6APAVere0/k1VfX/I/TxO7105j9A7d0kDStWebvlKkgaV5EZgfVWd17ov0ijxlo0kTYMkhwC/BJzEj8+NkbQHBhJJmh5/Rm8i7Pur6hutOyONGm/ZSJKk5pzUKkmSmjOQSJKk5gwkkiSpuZGb1HrQQQfV4sWLW3dDkiQN4dvf/va2qjpoV/WRCySLFy9m48aNrbshSZKGkOTh3dW9ZSNJkpozkEiSpOZG7paNJElzyc6dOxmVzwxLwrx5UxvrMJBIkrQf2rZtG2NjYzzxxBOtuzKUBQsWsGTJEg488MChnmcgkSRpPzQ2NsbTnvY0Fi1aRJLW3RlIVbF582bGxsZYvnz5UM81kEiStJ/ZuXMnTzzxBIsWLWL+/NF6qV60aBFbtmxh586dQ92+cVKrJEn7mfE5I6MyMtJvvM/DznsZKJAkeUaSu/uWtUm2Jzk0yWFJbk6yLsk9SY7ve97BSa5Osr57zil9tXlJLklyf1c/e6ieS5KkWWOgcaCqehR48fh6kvOBE6pqS5Irgdur6qQkK4DrkxxVVduB84HHq2p5kmXAbUluqapHgDOAY4GjgUOAu5J8rqrum84TlCRptrjib6/YJ/t9+8+/fZ/sdxhTvTH1VuA93eNTgWUAVXVHkk3A8cDngdOAt3S1B5KsAU4Grupql1fVDmBLkmuB04H3T7FPe22Q/9D7w380SZJmm6HnkCR5BbAI+Iski4B5VdX/cbAbgCXd4yXAg1OoSZKk/chFF13EypUrf7j+6KOP8sxnPpMtW7ZMy/6nMqn1t4BPdLdkACbOWpk4A6emWOttTM5LsnF82bp169AdliRJe+ess87ihhtu4Hvf+x4AH/vYxzj55JM59NBDp2X/QwWSJE+ld6vlSoCq2txt7//63SOBse7xGLB0CrUfqqqLq+qI8WXhwoXDdFmSJE2DZzzjGbzhDW/gqquuoqq47LLLOOecc6Zt/8OOkPxb4KsTJp5eB6wC6Ca1Hg7cOkltGXACcGNfbWWSA5IcSi/oXDOVk5AkSfveueeey2WXXcZf/uVf8qxnPYuXvOQl07bvYSe1vg342IRtFwCrk6wDtgFn9t3OuQi4Msl6YCewqqrGbzatBlYAa8fbVtW9w56AJEmaGccccwxLly7lne98Jx/4wAemdd9DBZKq+sVJtm0CTtxF+8fojXxMVttBN3oiSZJGw1lnncU555zDG9/4xmnd72h9Hq0kSXPY/vDRE5/97Gc5++yzWbBgwbTu14+OlyRJe/Sd73yHY445hrvvvpt3vetd075/R0gkSdIePfvZz+a++/bdh6k7QiJJkpozkEiStJ+Z6jfm7g+m+k3F3rKRJGk/M2/ePBYsWMDmzZtZtGjR0C/urVQVmzdvZsGCBcybN9yYh4FEkqT90JIlSxgbG5u274qZKQsWLGDJkuG/ms5AIknSfujAAw9k+fLl7Ny5c2Ru3SQZemRknIFEkqT92FRf4EfN3DhLSZK0XzOQSJKk5gwkkiSpOQOJJElqzkAiSZKaM5BIkqTmDCSSJKk5A4kkSWrOQCJJkpozkEiSpOYMJJIkqTkDiSRJas5AIkmSmjOQSJKk5gwkkiSpuYEDSZKDkvxhknVJvp7kk932w5Lc3G2/J8nxfc85OMnVSdYnWZvklL7avCSXJLm/q589vacmSZJGxfwh2l4I7ASOrqpK8lN922+vqpOSrACuT3JUVW0Hzgcer6rlSZYBtyW5paoeAc4AjgWOBg4B7kryuaq6b7pOTpIkjYaBRkiSPBV4K/DuqiqAqvpuVz4VuLTbdgewCRgfJTmtr/YAsAY4ua92eVXtqKotwLXA6Xt7QpIkafQMesvmKGAz8N4kdyb5QpJXJ1kEzKuqh/vabgCWdI+XAA9OoSZJkuaQQQPJAuC5wN9V1cuAc4A/pnfLpya0zYT1mmKttzE5L8nG8WXr1q0DdlmSJI2KQQPJg/Tmj3wKoKq+AjwA/AxAksV9bY8ExrrHY8DSKdR+qKourqojxpeFCxcO2GVJkjQqBgokVfUPwGeB1wAkORJYBnwDuA5Y1W1fARwO3No9tb+2DDgBuLGvtjLJAUkOpTen5Jq9PyVJkjRqhnmXzTuAK5P8D2AH8Paq+m6SC4DVSdYB24Azu3fYAFzUPWc9vRGWVd0EVoDVwApg7Xjbqrp3L89HkiSNoIEDSVV9E3jlJNs3ASfu4jmP0Rv5mKy2g270RJIkzW1+UqskSWrOQCJJkpozkEiSpOYMJJIkqTkDiSRJas5AIkmSmjOQSJKk5gwkkiSpOQOJJElqzkAiSZKaM5BIkqTmDCSSJKk5A4kkSWrOQCJJkpozkEiSpOYMJJIkqTkDiSRJas5AIkmSmjOQSJKk5gwkkiSpOQOJJElqzkAiSZKaM5BIkqTmDCSSJKm5gQNJkg1J7ktyd7ec1m0/LMnNSdYluSfJ8X3POTjJ1UnWJ1mb5JS+2rwklyS5v6ufPb2nJkmSRsX8Idu/sarumbDtQuD2qjopyQrg+iRHVdV24Hzg8apanmQZcFuSW6rqEeAM4FjgaOAQ4K4kn6uq+/bulCRJ0qiZjls2pwKXAlTVHcAmYHyU5LS+2gPAGuDkvtrlVbWjqrYA1wKnT0N/JEnSiBk2kHwqydeSfDTJ4iSLgHlV9XBfmw3Aku7xEuDBKdR+KMl5STaOL1u3bh2yy5IkaX83TCD5par6OeClwGbg4932mtAuE9ZrirVeg6qLq+qI8WXhwoVDdFmSJI2CgQNJVY11P58APgT8YlVtBkiyuK/pkcBY93gMWDqFmiRJmkMGCiRJnprkGX2b3gR8uXt8HbCqa7cCOBy4dZLaMuAE4Ma+2sokByQ5lN6ckmumfCaSJGlkDfoum2cBn05yAL1bK98E3tzVLgBWJ1kHbAPO7N5hA3ARcGWS9cBOYFU3gRVgNbACWDvetqru3auzkSRJI2mgQFJV3wResovaJuDEXdQeozfyMVltB93oiSRJmtv8pFZJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzRlIJElScwYSSZLUnIFEkiQ1ZyCRJEnNGUgkSVJzBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzRlIJElScwYSSZLUnIFEkiQ1ZyCRJEnNDRVIkvyXJJXkBd36YUluTrIuyT1Jju9re3CSq5OsT7I2ySl9tXlJLklyf1c/e/pOSZIkjZr5gzZM8lLg5cBY3+YLgdur6qQkK4DrkxxVVduB84HHq2p5kmXAbUluqapHgDOAY4GjgUOAu5J8rqrum6bzkiRJI2SgEZIkBwGXAmcD1Vc6tdtOVd0BbALGR0lO66s9AKwBTu6rXV5VO6pqC3AtcPpenYkkSRpZg96y+W/AJ7tgAUCSRcC8qnq4r90GYEn3eAnw4BRqkiRpjtljIEnyCmAF8EeTlGti893Uh6n1H/+8JBvHl61bt+6py5IkacQMMkJyAnAM8ECSDcARwGeA4wCSLO5reyQ/mmMyBiydQu1JquriqjpifFm4cOEAXZYkSaNkj4Gkqi6sqmdX1dKqWgpsBF5TVf8XuA5YBdBNaj0cuLV7an9tGb1gc2NfbWWSA5IcSm9OyTXTdlaSJGmkDPwum124AFidZB2wDTize4cNwEXAlUnWAzuBVd0EVoDV9G4DrR1vW1X37mVfJEnSiBo6kHSjJOOPNwEn7qLdY/RGPiar7aAbPZEkSfKTWiVJUnMGEkmS1JyBRJIkNWcgkSRJzRlIJElScwYSSZLUnIFEkiQ1ZyCRJEnNGUgkSVJzBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzRlIJElScwYSSZLUnIFEkiQ1ZyCRJEnNGUgkSVJzBhJJktScgUSSJDVnIJEkSc0NHEiS/FWSrya5O8kXkry4235YkpuTrEtyT5Lj+55zcJKrk6xPsjbJKX21eUkuSXJ/Vz97Ws9MkiSNjPlDtD21qh4FSPLrwJXAS4ELgdur6qQkK4DrkxxVVduB84HHq2p5kmXAbUluqapHgDOAY4GjgUOAu5J8rqrum66TkyRJo2HgEZLxMNI5BNjZPT4VuLRrcwewCRgfJTmtr/YAsAY4ua92eVXtqKotwLXA6VM6C0mSNNKGGSEhySeAV3WrJyVZBMyrqof7mm0AlnSPlwAPDlF72TD9kSRJs8NQk1qr6s1V9RzgvcBF45snNMvEp02x1tuYnJdk4/iydevWYbosSZJGwJTeZVNVH+dHIyUkWdxXPhIY6x6PAUunUOs/1sVVdcT4snDhwql0WZIk7ccGCiRJnp7k2X3rrwc2A1uA64BV3fYVwOHArV3T/toy4ATgxr7ayiQHJDmU3pySa/b2hCRJ0ugZdA7JIcCnk/wEvcmsDwOvrapKcgGwOsk6YBtwZvcOG+jd1rkyyfrueau6CawAq4EVwNrxtlV1796fkiRJGjUDBZKq+hZw3C5qm4ATd1F7jN7Ix2S1HXSjJ5IkaW7zk1olSVJzBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzRlIJElScwYSSZLUnIFEkiQ1ZyCRJEnNGUgkSVJzBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzRlIJElScwYSSZLUnIFEkiQ1N1AgSfKUJDckWZvk7iQ3J1na1Q7r1tcluSfJ8X3POzjJ1UnWd889pa82L8klSe7v6mdP+9lJkqSRMMwIyRXA86vqxcBfdOsAFwK3V9XzgLcCn0oyv6udDzxeVcuB1wB/lOQnu9oZwLHA0cBxwO8mOWZvTkaSJI2mgQJJVf2gqm6qquo23Q48t3t8KnBp1+4OYBMwPkpyWl/tAWANcHJf7fKq2lFVW4BrgdP37nQkSdIomuocknOBP0+yCJhXVQ/31TYAS7rHS4AHp1CTJElzyNCBJMm7gecB7+k21cQmE9ZrirXx452XZOP4snXr1mG7LEmS9nNDBZIk5wOnAL9SVf9UVZu77Yv7mh0JjHWPx4ClU6j9UFVdXFVHjC8LFy4cpsuSJGkEDBxIkpwHvAn411X1aF/pOmBV12YFcDhw6yS1ZcAJwI19tZVJDkhyKL05JddM+UwkSdLImr/nJpDkCOCDwDeBW5JA790zvwBcAKxOsg7YBpxZVdu7p14EXJlkPbATWNVNYAVYDawA1o63rap7p+GcJEnSiBkokFTVRnYxx6OqNgEn7qL2GL2Rj8lqO+hGTyRJ0tzmJ7VKkqTmDCSSJKk5A4kkSWrOQCJJkpozkEiSpOYMJJIkqTkDiSRJas5AIkmSmjOQSJKk5gwkkiSpOQOJJElqzkAiSZKaM5BIkqTmDCSSJKk5A4kkSWrOQCJJkpozkEiSpOYMJJIkqTkDiSRJas5AIkmSmjOQSJKk5gwkkiSpOQOJJElqzkAiSZKaGyiQJPlwkg1JKskL+rYfluTmJOuS3JPk+L7awUmuTrI+ydokp/TV5iW5JMn9Xf3s6T0tSZI0SgYdIbkeOB54cML2C4Hbq+p5wFuBTyWZ39XOBx6vquXAa4A/SvKTXe0M4FjgaOA44HeTHDP105AkSaNsoEBSVWuqauMkpVOBS7s2dwCb6AUXgNP6ag8Aa4CT+2qXV9WOqtoCXAucPtWTkCRJo23Kc0iSLALmVdXDfZs3AEu6x0t48ojKoDVJkjTH7O2k1pqwnt3Uh6n9qJCcl2Tj+LJ169YpdFOSJO3PphxIqmozQJLFfZuPBMa6x2PA0inUJh7n4qo6YnxZuHDhVLssSZL2U/P33GS3rgNWAe9PsgI4HLh1Qu0tSZYBJwDv6KutTPInwCH05pSctJd9kSRJu3HF316xxzZv//m3z0BPftxAgSTJpfQmpB4O/HWSrd27Zy4AVidZB2wDzqyq7d3TLgKuTLIe2Ams6iawAqwGVgBrx9tW1b3TckaSJGnkDBRIqmoVvdGOids3ASfu4jmP0Rv5mKy2Y7L9SZKkuclPapUkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzRlIJElScwYSSZLUnIFEkiQ1ZyCRJEnNGUgkSVJzBhJJktScgUSSJDVnIJEkSc0ZSCRJUnMGEkmS1JyBRJIkNWcgkSRJzRlIJElScwYSSZLUnIFEkiQ1ZyCRJEnNGUgkSVJzBhJJktRc00CS5HlJvphkbZIvJTm2ZX8kSVIbrUdI/jdwRVUdDXwA+Fjj/kiSpAaaBZIkhwEvBT7Zbfo0sCzJ0lZ9kiRJbbQcIXkO8J2q2g5QVQWMAUsa9kmSJDUwv/Hxa8J6JjZIch5wXt+mHUke2kf9WQhs3V2DlazcR4eeU/Z4nTUtvM4zw+s8c7zWM2AlK/fVdV68u2J6AxMzr7tlsw5YVFXbkwT4LvDyqtrQqE8bq+qIFseeS7zOM8PrPDO8zjPHaz0zWl3nZrdsqurvgS8DZ3Sb3gBsaBVGJElSO61v2awErkrybuD7wL9r3B9JktRA00BSVd8AXtGyDxNc3LoDc4TXeWZ4nWeG13nmeK1nRpPr3GwOiSRJ0rjWH4wmSZJkIJEkSe3NuUAy6PfnJHlbknVJ7k9yRZLWE4BHyiDXOckvJ/mbJH+X5J4kv9+9/VsDGub7oJI8pbvWd85kH2eDIf5uvDDJ55Pcm+QbSU6Z6b6OsgH/biTJRUm+nuSrSW5JsrxFf0dVkg8n2ZCkkrxgN+1m9HVwzgUSBvj+nCTLgN8DjgeWA4cDb5vJTs4Cg3xP0SPAm6rqWOBlwAnAm2aui7PCMN8H9fvAbTPSq9lnkL8bBwM3AO+tqp8Bfhb4wkx2chYY5Pf514BfAl5cVS8CPgv8wcx1cVa4nt7r24O7atDidXBOBZIhvj/njcCfVtWm7iPtL8cXyoENep2r6stV9c3u8Q+Au4HnzlxPR9sw3weV5BeB5wGrZ6yDs8QQ1/k3gNuq6laAqtpeVQ/PWEdH3JDfb3YQ8JRuRPXpwMYZ6eQsUVVrqmpP12zGXwfnVCBh8O/PWcKTk+OGSdpo14b+nqIkh9P7B3DTjPRwdhjoOid5KvAh4J0z3cFZYtDf52OBHyT5iyR3J/lEkt1+VLaeZNDr/OfALcBD9D7d+9XA+2awn3PFjL8OzrVAAgN8f84k7ZzXMLxBrzNJnk7vj8wHququfdqr2WeQ63wRcGlVfXsG+jNbDXKdFwCvofeBjy8BvgVcuo/7NdsMcp1fChwD/DTwbHq3bP5wH/drrprR18G5Fki+BRwxPjGnG+57Dr0U3m8MWNq3fuQkbbRrg15nkjwNuBm4sar80KPhDHqdjwfel2QD8MfAC5N8fSY7OuIGvc4PArdU1be7/7v/FHDcjPZ0tA16nd9C7zo/WlU7gY8Dr5rJjs4RM/46OKcCyRDfn/Np4PVJntX9o3gHvT/kGsCg1znJQnph5DNV9Xsz2slZYNDrXFUvqqqlVbUUOB34WlX97Ez2dZQN8XfjWmBFN+IHcBLwlRnp5CwwxHX+JvDqJAu69dcB98xIJ+eWmX8drKo5tQDPp/dOg7XAncDPdts/CvxaX7uzgPX0fvk/Cixo3fdRWga5zsB7gCfoTWYdX97Tuu+jtAz6+9zX/pXAna37PWrLEH833gx8nV4QuQk4onXfR2kZ8O/GQcBHgPuArwKfAZa27vsoLfRuJW4EttObi7N+4nXu1mf0ddCPjpckSc3NqVs2kiRp/2QgkSRJzRlIJElScwYSSZLUnIFEkiQ1ZyCRJEnNGUgkSVJzBhJJktScgUSSJDX3/wHlD1zON3W60wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x320 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "## Data Clean"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Initliazed parameters"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cv</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>data_cleaner_args</td>\n",
       "      <td>{'nan_chars': None, 'correct_object_dtype': Tr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>name</td>\n",
       "      <td>data_clean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train_test_split_strategy</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         key  \\\n",
       "0                         cv   \n",
       "1          data_cleaner_args   \n",
       "2                       name   \n",
       "3  train_test_split_strategy   \n",
       "\n",
       "                                               value  \n",
       "0                                               True  \n",
       "1  {'nan_chars': None, 'correct_object_dtype': Tr...  \n",
       "2                                         data_clean  \n",
       "3                                               None  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Fitted parameters"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>input_features</td>\n",
       "      <td>[id, age, job, marital, education, default, ba...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>selected_features</td>\n",
       "      <td>[id, age, job, marital, education, default, ba...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unselected_features</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>X_train.shape</td>\n",
       "      <td>(8000, 17)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>y_train.shape</td>\n",
       "      <td>(8000,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>X_eval.shape</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>y_eval.shape</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>X_test.shape</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>unselected_reason</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>kept/dropped feature count</td>\n",
       "      <td>17/0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          key  \\\n",
       "0              input_features   \n",
       "1           selected_features   \n",
       "2         unselected_features   \n",
       "3               X_train.shape   \n",
       "4               y_train.shape   \n",
       "5                X_eval.shape   \n",
       "6                y_eval.shape   \n",
       "7                X_test.shape   \n",
       "8           unselected_reason   \n",
       "9  kept/dropped feature count   \n",
       "\n",
       "                                               value  \n",
       "0  [id, age, job, marital, education, default, ba...  \n",
       "1  [id, age, job, marital, education, default, ba...  \n",
       "2                                                 []  \n",
       "3                                         (8000, 17)  \n",
       "4                                            (8000,)  \n",
       "5                                               None  \n",
       "6                                               None  \n",
       "7                                               None  \n",
       "8                                                 {}  \n",
       "9                                               17/0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Elapsed"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "* 0.057 seconds"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "## Space Searching"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Initliazed parameters"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cv</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>name</td>\n",
       "      <td>space_searching</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>num_folds</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         key            value\n",
       "0         cv             True\n",
       "1       name  space_searching\n",
       "2  num_folds                3"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### Experiment Settings:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "HyperDT(searcher=EvolutionSearcher(space_fn=<function mini_dt_space at 0x7fb00459fcb0>, population_size=30, sample_size=10, regularized=True, optimize_direction='max', space_sample_validation_fn=<function mini_dt_space_validator at 0x7fb00459fd40>, random_state=RandomState(MT19937) at 0x7FB003CA2050), callbacks=[EarlyStoppingCallback(max_no_improvement_trials=10, mode='max', time_limit=3599.9423274993896), NotebookCallback(), ProgressiveCallback()], reward_metric='Precision', max_model_size=None)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X</th>\n",
       "      <th>y</th>\n",
       "      <th>X_eval</th>\n",
       "      <th>y_eval</th>\n",
       "      <th>cv</th>\n",
       "      <th>num_folds</th>\n",
       "      <th>max_trials</th>\n",
       "      <th>fit_kwargs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(8000, 17)</td>\n",
       "      <td>(8000,)</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>(pos_label, verbose, n_jobs)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            X        y X_eval y_eval    cv  num_folds  max_trials  \\\n",
       "0  (8000, 17)  (8000,)   None   None  True          3           5   \n",
       "\n",
       "                     fit_kwargs  \n",
       "0  (pos_label, verbose, n_jobs)  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### Trials Summary:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Trial No.</th>\n",
       "      <th>Previous reward</th>\n",
       "      <th>Best trial</th>\n",
       "      <th>Best reward</th>\n",
       "      <th>Total elapsed</th>\n",
       "      <th>Valid trials</th>\n",
       "      <th>Max trials</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>0.120317</td>\n",
       "      <td>2</td>\n",
       "      <td>0.507937</td>\n",
       "      <td>33.065014</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Trial No.  Previous reward  Best trial  Best reward  Total elapsed  \\\n",
       "0          5         0.120317           2     0.507937      33.065014   \n",
       "\n",
       "   Valid trials  Max trials  \n",
       "0             5           5  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### Best Trial:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "<thead>\n",
       "<tr style=\"text-align: right;\">\n",
       "  <th>key</th>\n",
       "  <th>value</th>\n",
       "</tr>\n",
       "</thead>\n",
       "<tbody><tr>\n",
       "  <td>signature</td>\n",
       "  <td>9fa4b26cd239f2a4900fa82e43d06519</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <td>vectors</td>\n",
       "  <td>[5, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0]</td>\n",
       "</tr><tr>\n",
       "  <td>0-Module_DTModuleSpace_1.nets</td>\n",
       "  <td>['dnn_nets', 'fm_nets']</td>\n",
       "</tr>\n",
       "<tr><tr>\n",
       "  <td>1-Module_DTModuleSpace_1.auto_categorize</td>\n",
       "  <td>False</td>\n",
       "</tr>\n",
       "<tr><tr>\n",
       "  <td>2-Module_DTModuleSpace_1.cat_remain_numeric</td>\n",
       "  <td>True</td>\n",
       "</tr>\n",
       "<tr><tr>\n",
       "  <td>3-Module_DTModuleSpace_1.auto_discrete</td>\n",
       "  <td>False</td>\n",
       "</tr>\n",
       "<tr><tr>\n",
       "  <td>4-Module_DTModuleSpace_1.apply_gbm_features</td>\n",
       "  <td>True</td>\n",
       "</tr>\n",
       "<tr><tr>\n",
       "  <td>5-Module_DTModuleSpace_1.gbm_feature_type</td>\n",
       "  <td>dense</td>\n",
       "</tr>\n",
       "<tr><tr>\n",
       "  <td>6-Module_DTModuleSpace_1.embeddings_output_dim</td>\n",
       "  <td>4</td>\n",
       "</tr>\n",
       "<tr><tr>\n",
       "  <td>7-Module_DTModuleSpace_1.embedding_dropout</td>\n",
       "  <td>0</td>\n",
       "</tr>\n",
       "<tr><tr>\n",
       "  <td>8-Module_DTModuleSpace_1.stacking_op</td>\n",
       "  <td>add</td>\n",
       "</tr>\n",
       "<tr><tr>\n",
       "  <td>9-Module_DTModuleSpace_1.output_use_bias</td>\n",
       "  <td>False</td>\n",
       "</tr>\n",
       "<tr><tr>\n",
       "  <td>10-Module_DTModuleSpace_1.apply_class_weight</td>\n",
       "  <td>False</td>\n",
       "</tr>\n",
       "<tr><tr>\n",
       "  <td>11-Module_DTModuleSpace_1.earlystopping_patience</td>\n",
       "  <td>3</td>\n",
       "</tr>\n",
       "<tr><tr>\n",
       "  <td>12-Module_DTFit_1.batch_size</td>\n",
       "  <td>128</td>\n",
       "</tr>\n",
       "<tr><tr>\n",
       "  <td>13-Module_DnnModule_1.hidden_units</td>\n",
       "  <td>200</td>\n",
       "</tr>\n",
       "<tr><tr>\n",
       "  <td>14-Module_DnnModule_1.reduce_factor</td>\n",
       "  <td>1</td>\n",
       "</tr>\n",
       "<tr><tr>\n",
       "  <td>15-Module_DnnModule_1.dnn_dropout</td>\n",
       "  <td>0.3</td>\n",
       "</tr>\n",
       "<tr><tr>\n",
       "  <td>16-Module_DnnModule_1.use_bn</td>\n",
       "  <td>False</td>\n",
       "</tr>\n",
       "<tr>  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "DAG_HyperSpace_1"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### Top trials:"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Trial No.</th>\n",
       "      <th>Reward</th>\n",
       "      <th>Elapsed</th>\n",
       "      <th>Space Vector</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>0.507937</td>\n",
       "      <td>6.072648</td>\n",
       "      <td>[5, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>0.361022</td>\n",
       "      <td>8.402343</td>\n",
       "      <td>[5, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>0.321230</td>\n",
       "      <td>6.882224</td>\n",
       "      <td>[4, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 2, 1, 0, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>0.120317</td>\n",
       "      <td>4.304229</td>\n",
       "      <td>[2, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0.092308</td>\n",
       "      <td>7.289087</td>\n",
       "      <td>[3, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 2, 1, 1, 1, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Trial No.    Reward   Elapsed  \\\n",
       "0          2  0.507937  6.072648   \n",
       "1          3  0.361022  8.402343   \n",
       "2          4  0.321230  6.882224   \n",
       "3          5  0.120317  4.304229   \n",
       "4          1  0.092308  7.289087   \n",
       "\n",
       "                                        Space Vector  \n",
       "0  [5, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, ...  \n",
       "1  [5, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, ...  \n",
       "2  [4, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 2, 1, 0, 1, ...  \n",
       "3  [2, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, ...  \n",
       "4  [3, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 2, 1, 1, 1, ...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "search:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:AutoGraph could not transform <bound method MultiColumnEmbedding.call of <deeptables.models.layers.MultiColumnEmbedding object at 0x7fb0a0fb8a90>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: module 'gast' has no attribute 'Index'\n",
      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
      "WARNING: AutoGraph could not transform <bound method MultiColumnEmbedding.call of <deeptables.models.layers.MultiColumnEmbedding object at 0x7fb0a0fb8a90>> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: module 'gast' has no attribute 'Index'\n",
      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### Fitted parameters"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>input_features</td>\n",
       "      <td>[id, age, job, marital, education, default, ba...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>best_reward</td>\n",
       "      <td>0.507937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>history</td>\n",
       "      <td>TrialHistory(direction='max', size=5, succeede...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              key                                              value\n",
       "0  input_features  [id, age, job, marital, education, default, ba...\n",
       "1     best_reward                                           0.507937\n",
       "2         history  TrialHistory(direction='max', size=5, succeede..."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Elapsed"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "* 33.111 seconds"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "## Final Ensemble"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Initliazed parameters"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ensemble_size</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>name</td>\n",
       "      <td>final_ensemble</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>scorer</td>\n",
       "      <td>make_scorer(precision_score, average=binary, p...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             key                                              value\n",
       "0  ensemble_size                                                 20\n",
       "1           name                                     final_ensemble\n",
       "2         scorer  make_scorer(precision_score, average=binary, p..."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Fitted parameters"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>weights</td>\n",
       "      <td>[0.85, 0.05, 0.0, 0.05, 0.05]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>scores</td>\n",
       "      <td>[0.7692307692307693, 0.7692307692307693, 0.785...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>best_stack</td>\n",
       "      <td>[0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 3, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>hits</td>\n",
       "      <td>{0: 17, 1: 1, 4: 1, 3: 1}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ensemble_size</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "GreedyEnsemble(weight=[0.85 0.05 0.   0.05 0.05], scores=[0.7692307692307693, 0.7692307692307693, 0.7857142857142857, 0.9142857142857143, 0.9, 0.88, 0.9285714285714286, 0.9285714285714286, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Elapsed"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "* 6.056 seconds"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "experiment=make_experiment(train_data.copy(), target='y', reward_metric='Precision', pos_label='yes', max_trials=5)\n",
    "estimator=experiment.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate the estimator with classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "          no    0.89050   1.00000   0.94208      1781\n",
      "         yes    0.00000   0.00000   0.00000       219\n",
      "\n",
      "    accuracy                        0.89050      2000\n",
      "   macro avg    0.44525   0.50000   0.47104      2000\n",
      "weighted avg    0.79299   0.89050   0.83892      2000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "X_test = test_data.copy()\n",
    "y_test = X_test.pop('y')\n",
    "y_pred = estimator.predict(X_test)\n",
    "print(classification_report(y_test, y_pred, digits=5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export Estimator with Hypernets utility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from hypernets.utils import load_estimator, save_estimator\n",
    "\n",
    "save_estimator(estimator,'dt_automl_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
